package edu.usc.utils;

import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;

import opennlp.tools.postag.POSModel;
import opennlp.tools.postag.POSTaggerME;

import com.aliasi.tokenizer.PorterStemmerTokenizerFactory;

public class Parse {

	public static String folderURL = "C:\\Ramesh\\works\\projects\\java_ws\\StorySummerizer\\trainingsummary";
	//public static String folderURL = "C:\\Ramesh\\works\\projects\\java_ws\\StorySummerizer\\test";
	public static String stemmedToCompleteURL = "C:\\Ramesh\\works\\projects\\java_ws\\StorySummerizer\\resources\\stemmedToComplete.data";
	private static String arffURL = "temp_2_6_class.arff";
	public static HashMap<String, Integer> wordCounts = new HashMap<>();
	public static HashMap<String, Integer> sentenceCounts = new HashMap<>();

	public static HashMap<String, Double> wordProbs = new HashMap<>();
	public static HashMap<String, Double> sentenceProbs = new HashMap<>();

	public static HashMap<String, Integer> wordPOCCounts = new HashMap<>();
	public static HashMap<String, Double> wordPOCProbs = new HashMap<>();
	public static HashMap<String, Integer> sentencePOCCounts = new HashMap<>();
	public static HashMap<String, Double> sentencePOCProbs = new HashMap<>();

	public static HashMap<String, Integer> POCCounts = new HashMap<>();
	public static HashMap<String, Integer> POCPOCCounts = new HashMap<>();
	public static HashMap<String, Double> POCProbs = new HashMap<>();
	public static HashMap<String, Double> POCPOCProbs = new HashMap<>();

	public static HashMap<String, HashSet<String>> wordsPOCs = new HashMap<>();
	public static HashMap<String, Integer>	wordWord = new HashMap<>();
	public static HashMap<String, ArrayList<String>>	wordWordPOC = new HashMap<>();
	private static ArrayList<String> temp = new ArrayList<>();
	public static void main(String[] args) {
		FileUtils.writeToFileNew(stemmedToCompleteURL, "");
		populate(folderURL);

		/*System.out.println(wordCounts);
		System.out.println(wordPOCCounts);
		System.out.println(POCCounts);
		System.out.println(wordPOCProbs);
		Double sum = 0.0;
		for(String key: wordPOCProbs.keySet()) {			
			if(key.split("\\|")[1].equals("ER")) {
				System.out.println(key + " :: " + wordPOCProbs.get(key));
				sum += wordPOCProbs.get(key);
			}
		}
		System.out.println(sum);
		sum = 0.0;
		System.out.println(POCProbs);
		for(String key: POCProbs.keySet()) {			
			System.out.println(key + " :: " + POCProbs.get(key));
			sum += POCProbs.get(key);
		}
		System.out.println(sum);
		sum = 0.0;
		for(String key: POCPOCProbs.keySet()) {			
			if(key.split("\\|")[1].equals("ER")) {
				System.out.println(key + " :: " + POCPOCProbs.get(key));
				sum += POCPOCProbs.get(key);
			}
		}
		System.out.println(sum);
		 */
		//System.out.println(sentenceCounts.size());
		//System.out.println(sentencePOCProbs);
		ArrayList<String> wordsOrder = new ArrayList<>();
		ArrayList<String> wordsWordsOrder = new ArrayList<>();
		for(String word: wordCounts.keySet()) {
			if(wordCounts.get(word) > 1) {
				System.out.println(word + "\t" + wordCounts.get(word) + "\t" + wordsPOCs.get(word));
				wordsOrder.add(word);
			}
		}
		//System.out.println(wordWord.size());
		for(String word: wordWord.keySet()) {
			if(wordWord.get(word) > 1) {
					System.out.println(word + "\t" + wordWord.get(word) + "\t" + wordWordPOC.get(word));
					wordsWordsOrder.add(word);
			}
		}

		createARFF(folderURL, wordsOrder, wordsWordsOrder);
	}

	public static void createARFF(String url,
			ArrayList<String> wordsOrder, 
			ArrayList<String> wordsWordsOrder) {
		InputStream modelIn = null;
		POSModel model = null;
		try {
		  modelIn = new FileInputStream("en-pos-maxent.bin");
		  model = new POSModel(modelIn);
		}
		catch (IOException e) {
		  // Model loading failed, handle the error
		  e.printStackTrace();
		}
		finally {
		  if (modelIn != null) {
		    try {
		      modelIn.close();
		    }
		    catch (IOException e) {
		    }
		  }
		}
		POSTaggerME tagger = new POSTaggerME(model);
		
		ArrayList<String> files = FileUtils.listAllFilesInDirectory(url);
		int totalSizeRow = (wordsOrder.size() + 
				wordsWordsOrder.size() + 25);
		int newF = 23;
		ArrayList<String> noneList = FileUtils.contentsOfTheFileAsList("None");
		ArrayList<String> othersList = FileUtils.contentsOfTheFileAsList("Others");
		
		ArrayList<String> ccList = FileUtils.contentsOfTheFileAsList("CC");
		ArrayList<String> crList = FileUtils.contentsOfTheFileAsList("CR");
		ArrayList<String> drList = FileUtils.contentsOfTheFileAsList("DR");
		ArrayList<String> erList = FileUtils.contentsOfTheFileAsList("ER");
		ArrayList<String> hrList = FileUtils.contentsOfTheFileAsList("HR");
		ArrayList<String> rmList = FileUtils.contentsOfTheFileAsList("RM");
		ArrayList<String> scList = FileUtils.contentsOfTheFileAsList("SC");
		ArrayList<String> suList = FileUtils.contentsOfTheFileAsList("SU");
		ArrayList<String> selfLList = FileUtils.contentsOfTheFileAsList("SelfL");
		ArrayList<String> selfRList = FileUtils.contentsOfTheFileAsList("SelfR");
		ArrayList<String> socLList = FileUtils.contentsOfTheFileAsList("SocL");
		
		
		//System.out.println(files);
		ArrayList<ArrayList<Integer>> toARFF = new ArrayList<>();
		for(String file: files) {
			ArrayList<String> contents = FileUtils.contentsOfTheFileAsList(file);
			String prevPOC = "<s>";
			for(String line: contents) {
				ArrayList<Integer> lineI = new ArrayList<>(totalSizeRow);
				for(int i=0; i < totalSizeRow; i++) {
					lineI.add(0);
				}
				String[] mods = line.split("\t");
				String cleanSentence = FileUtils.cleanSentences(mods[1]);
				// ADDING THE STOPPING AND STEMMING HERE: COMMENT OUT IF UNNECESSARY
				int myCounts = 0;
				int iCounts = 0;
				int jCounts = 0;
				int nCounts = 0;
				int vCounts = 0;
				int rCounts = 0;
				int dpcCounts = 0;
				int oCounts = 0;
				
				String[] tempMy = cleanSentence.split("my");
				String[] tempI = cleanSentence.split("I");
				myCounts = tempMy.length - 1;
				iCounts = tempI.length - 1;
				
				int noneCounts = 0;
				int otherCounts = 0;
				
				int ccCounts = 0;
				int crCounts = 0;
				int drCounts = 0;
				int erCounts = 0;
				int hrCounts = 0;
				int scCounts = 0;
				int suCounts = 0;
				int rmCounts = 0;
				int socLCounts = 0;
				int selfLCounts = 0;
				int selfRCounts = 0;
				
				String[] tags = tagger.tag(cleanSentence.split(" "));
				for(int i=0; i < tags.length; i++) {
					if(tags[i].charAt(0) == 'J') {
						jCounts++;
					} else if(tags[i].charAt(0) == 'N') {
						nCounts++;
					}  else if(tags[i].charAt(0) == 'V') {
						vCounts++;
					}  else if(tags[i].charAt(0) == 'R') {
						rCounts++;
					}  else if(tags[i].charAt(0) == 'D' || 
							tags[i].charAt(0) == 'P' || 
							tags[i].charAt(0) == 'C') {
						dpcCounts++;
					}  else {
						oCounts++;
					}
				}
				int diff = cleanSentence.length();
				int len = diff;
				String stoppedTemp = 
						FileUtils.removeStopWordsString(cleanSentence, null);
				stoppedTemp = stoppedTemp.replaceAll("[^\\p{Alpha} ]", "");
				String stemmedTemp =
						FileUtils.stemString(stoppedTemp);
				cleanSentence = stemmedTemp;
				String prev = "_BEGIN_";
				diff = diff - cleanSentence.length();
				// COMMENT TILL HERE.
				String[] tokens = cleanSentence.split(" ");				
				for(String token: tokens) {
					
					if(noneList.contains(token)) {
						noneCounts++;
					}
					
					if(othersList.contains(token)) {
						otherCounts++;
					}
					//==
					if(ccList.contains(token)) {
						ccCounts++;
					}
					if(crList.contains(token)) {
						crCounts++;
					}
					if(erList.contains(token)) {
						erCounts++;
					}
					if(drList.contains(token)) {
						drCounts++;
					}
					if(hrList.contains(token)) {
						hrCounts++;
					}
					if(scList.contains(token)) {
						scCounts++;
					}
					if(suList.contains(token)) {
						suCounts++;
					}
					if(rmList.contains(token)) {
						rmCounts++;
					}
					if(selfLList.contains(token)) {
						selfLCounts++;
					}
					if(selfRList.contains(token)) {
						selfRCounts++;
					}
					if(socLList.contains(token)) {
						socLCounts++;
					}
					//==
					int indexW = wordsOrder.indexOf(token);
					if(indexW != -1) {
						lineI.set(indexW, wordCounts.get(token));
					}
					
					int indexWW = wordsWordsOrder.indexOf(token+"|"+prev);
					if(indexWW != -1) {
						lineI.set(wordsOrder.size()+indexWW, wordWord.get(token+"|"+prev));
					}
				}
				
				if(prevPOC.equalsIgnoreCase("None")) {
					lineI.set(wordsOrder.size() + wordsWordsOrder.size(), 0);
				} else if (prevPOC.equalsIgnoreCase("CR")) {
					lineI.set(wordsOrder.size() + wordsWordsOrder.size(), 1);
				} else if (prevPOC.equalsIgnoreCase("ER")) {
					lineI.set(wordsOrder.size() + wordsWordsOrder.size(), 2);
				} else if (prevPOC.equalsIgnoreCase("DR")) {
					lineI.set(wordsOrder.size() + wordsWordsOrder.size(), 3);
				} else if (prevPOC.equalsIgnoreCase("HR")) {
					lineI.set(wordsOrder.size() + wordsWordsOrder.size(), 4);
				} else if (prevPOC.equalsIgnoreCase("SocL")) {
					lineI.set(wordsOrder.size() + wordsWordsOrder.size(), 5);
				} else if (prevPOC.equalsIgnoreCase("SelfL")) {
					lineI.set(wordsOrder.size() + wordsWordsOrder.size(), 6);
				} else if (prevPOC.equalsIgnoreCase("SelfR")) {
					lineI.set(wordsOrder.size() + wordsWordsOrder.size(), 7);
				} else if (prevPOC.equalsIgnoreCase("SC")) {
					lineI.set(wordsOrder.size() + wordsWordsOrder.size(), 8);
				} else if (prevPOC.equalsIgnoreCase("SU")) {
					lineI.set(wordsOrder.size() + wordsWordsOrder.size(), 9);
				} else if (prevPOC.equalsIgnoreCase("CC")) {
					lineI.set(wordsOrder.size() + wordsWordsOrder.size(), 10);
				} else if (prevPOC.equalsIgnoreCase("RM")) {
					lineI.set(wordsOrder.size() + wordsWordsOrder.size(), 11);
				} else {
					lineI.set(wordsOrder.size() + wordsWordsOrder.size(), 12);
				}
				prevPOC = mods[2];
				// New features
				
				lineI.set(wordsOrder.size() + wordsWordsOrder.size()+1, len);
				lineI.set(wordsOrder.size() + wordsWordsOrder.size()+2, diff);
				
				lineI.set(wordsOrder.size() + wordsWordsOrder.size()+3, jCounts);
				lineI.set(wordsOrder.size() + wordsWordsOrder.size()+4, nCounts);
				lineI.set(wordsOrder.size() + wordsWordsOrder.size()+5, rCounts);
				lineI.set(wordsOrder.size() + wordsWordsOrder.size()+6, vCounts);
				lineI.set(wordsOrder.size() + wordsWordsOrder.size()+5, dpcCounts);
				lineI.set(wordsOrder.size() + wordsWordsOrder.size()+6, oCounts);
				lineI.set(wordsOrder.size() + wordsWordsOrder.size()+7, myCounts);
				lineI.set(wordsOrder.size() + wordsWordsOrder.size()+8, iCounts);
				lineI.set(wordsOrder.size() + wordsWordsOrder.size()+9, noneCounts);
				lineI.set(wordsOrder.size() + wordsWordsOrder.size()+10, otherCounts);
				
				lineI.set(wordsOrder.size() + wordsWordsOrder.size()+11, ccCounts);
				lineI.set(wordsOrder.size() + wordsWordsOrder.size()+12, crCounts);
				lineI.set(wordsOrder.size() + wordsWordsOrder.size()+13, scCounts);
				lineI.set(wordsOrder.size() + wordsWordsOrder.size()+14, drCounts);
				lineI.set(wordsOrder.size() + wordsWordsOrder.size()+15, hrCounts);
				lineI.set(wordsOrder.size() + wordsWordsOrder.size()+16, suCounts);
				lineI.set(wordsOrder.size() + wordsWordsOrder.size()+17, rmCounts);
				lineI.set(wordsOrder.size() + wordsWordsOrder.size()+18, erCounts);
				lineI.set(wordsOrder.size() + wordsWordsOrder.size()+19, selfLCounts);
				lineI.set(wordsOrder.size() + wordsWordsOrder.size()+20, selfRCounts);
				lineI.set(wordsOrder.size() + wordsWordsOrder.size()+21, socLCounts);
				
				if(mods[2].equals("CR") || mods[2].equals("ER") || mods[2].equals("DR")) {
					ArrayList<Integer> temp = new ArrayList<>(lineI); 
					temp.set(wordsOrder.size() + wordsWordsOrder.size()+1+newF, 1);
					toARFF.add(temp);
				}
				
				if(mods[2].equals("CR") || mods[2].equals("ER") || mods[2].equals("DR") 
						|| mods[2].equals("HR") || mods[2].equals("SocL")) {
					ArrayList<Integer> temp = new ArrayList<>(lineI); 
					temp.set(wordsOrder.size() + wordsWordsOrder.size()+1+newF, 2);
					//toARFF.add(temp);
				}
				
				if(mods[2].equals("HR") || mods[2].equals("SocL")) {
					ArrayList<Integer> temp = new ArrayList<>(lineI); 
					temp.set(wordsOrder.size() + wordsWordsOrder.size()+1+newF, 3);
					//toARFF.add(temp);
				}
				
				if(mods[2].equals("HR") || mods[2].equals("SocL") || 
						mods[2].equals("SelfL") || mods[2].equals("SelfR") || mods[2].equals("SC") || mods[2].equals("SU")
						|| mods[2].equals("CC") || mods[2].equals("RM")) {
					ArrayList<Integer> temp = new ArrayList<>(lineI); 
					temp.set(wordsOrder.size() + wordsWordsOrder.size()+1+newF, 1);
					toARFF.add(temp);
				}
				
				if(mods[2].equals("CC") || mods[2].equals("RM")) {
					ArrayList<Integer> temp = new ArrayList<>(lineI); 
					temp.set(wordsOrder.size() + wordsWordsOrder.size()+1+newF, 5);
					//toARFF.add(temp);	
				}
				
				if(mods[2].equalsIgnoreCase("None")) {
					ArrayList<Integer> temp = new ArrayList<>(lineI); 
					temp.set(wordsOrder.size() + wordsWordsOrder.size()+1+newF, 0);
					toARFF.add(temp);
				}
			}
		}
		//System.out.println(toARFF.size());
		
		FileUtils.writeToFileNew(arffURL, "@relation MyRelation");
		FileUtils.writeToFileAlreadyExisting(arffURL, "");
		FileUtils.writeToFileAlreadyExisting(arffURL, "");
		
		for(int i=0; i < totalSizeRow-1; i++) {
			FileUtils.writeToFileAlreadyExisting(arffURL, "@attribute F"+i+"F numeric");
		}
		//FileUtils.writeToFileAlreadyExisting(arffURL, "@attribute class {5,4,3,2,1,0}");
		FileUtils.writeToFileAlreadyExisting(arffURL, "@attribute class {1,0}");
		FileUtils.writeToFileAlreadyExisting(arffURL, "");
		FileUtils.writeToFileAlreadyExisting(arffURL, "@data");
		for(ArrayList<Integer> temp: toARFF) {
			//System.out.println(temp.size());
			String line = "";
			if(temp.size() != totalSizeRow) {
				System.out.println(temp.size()+"==============="+totalSizeRow);
			}
			for(int j=0; j < temp.size(); j++) {
				//System.out.print(i + " , ");
				if (j < (temp.size()-1)){
					line = line + temp.get(j) + ",";
				} else {
					line = line + temp.get(j);
				}
			}
			FileUtils.writeToFileAlreadyExisting(arffURL, line);
		}
	}
	
	public static void populate(String url) {
		ArrayList<String> files = FileUtils.listAllFilesInDirectory(url);
		//System.out.println(files);
		for(String file: files) {
			ArrayList<String> contents = FileUtils.contentsOfTheFileAsList(file);
			String prevPOC = "<s>";
			for(String line: contents) {
				String[] mods = line.split("\t");
				String cleanSentence = FileUtils.cleanSentences(mods[1]);
				// ADDING THE STOPPING AND STEMMING HERE: COMMENT OUT IF UNNECESSARY
				/*String stoppedTemp = 
						FileUtils.removeStopWordsString(cleanSentence, null);
				stoppedTemp = stoppedTemp.replaceAll("[^\\p{Alpha} ]", "");
				String stemmedTemp =
						FileUtils.stemString(stoppedTemp);
				cleanSentence = stemmedTemp;*/
				// COMMENT TILL HERE.
				String prev = "_BEGIN_";
				String[] tokens = cleanSentence.split(" ");				
				for(String token: tokens) {
					Integer count = wordCounts.get(token);
					Integer wPOCCount = wordPOCCounts.get(token+"|"+mods[2]);					
					Integer pocsCounts = POCCounts.get(mods[2]);
					// COMMENT THIS AS WELL: START
					Integer val = wordWord.get(token+"|"+prev);
					ArrayList<String> temps = wordWordPOC.get(token+"|"+prev);
					if(val == null) {
						val = 1;
					} else {
						val++;
					}
					
					if (temps == null) {
						temps = new ArrayList<>();
						temps.add(mods[2]);
					} else {
						temps.add(mods[2]);
					}
					wordWord.put(token+"|"+prev, val);
					wordWordPOC.put(token+"|"+prev, temps);
					prev = token;
					// COMMENT THIS AS WELL: END
					if(count == null) {
						count = 1; 
					} else if (count != null) {
						count++;
					}
					if(pocsCounts == null) {
						pocsCounts = 1;
					} else if(pocsCounts!=null) {
						pocsCounts++;
					}
					if(wPOCCount == null) {
						wPOCCount = 1;
					} else if (wPOCCount != null){						
						wPOCCount++;						
					}
					wordCounts.put(token, count);
					wordPOCCounts.put(token+"|"+mods[2], wPOCCount);
					POCCounts.put(mods[2], pocsCounts);

					HashSet<String> pocWord = wordsPOCs.get(token);
					if(pocWord == null) {
						pocWord = new HashSet<>();
						pocWord.add(mods[2]);						
					} else {
						pocWord.add(mods[2]);
					}
					wordsPOCs.put(token, pocWord);
				}
				Integer pocpocCounts = POCPOCCounts.get(mods[2]+"|"+prevPOC);

				if(pocpocCounts == null) {
					pocpocCounts = 1;
				} else if (pocpocCounts != null) {
					pocpocCounts++;
				}

				POCPOCCounts.put(mods[2]+"|"+prevPOC, pocpocCounts);
				prevPOC = mods[2];

				String stopped = 
						FileUtils.removeStopWordsString(cleanSentence, null);
				stopped = stopped.replaceAll("[^\\p{Alpha} ]", "");
				String stemmed =
						FileUtils.stemString(stopped);
				//FileUtils.writeToFileAlreadyExisting(stemmedToCompleteURL, cleanSentence+"\t"+stemmed);
				stemmed = stemmed.replaceAll("( )+", " ");

				temp.add(cleanSentence+"\t"+stemmed);
				Integer sentCounts = sentenceCounts.get(stemmed);
				Integer sentPOCCounts = sentencePOCCounts.get(stemmed+"|"+mods[2]);
				if(sentCounts == null) {
					sentCounts = 1;
				} else if (sentCounts != null) {
					sentCounts++;
				}
				if(sentPOCCounts == null) {
					sentPOCCounts = 1;
				} else if (sentPOCCounts != null) {
					sentPOCCounts++;
				}

				sentenceCounts.put(stemmed, sentCounts);
				sentencePOCCounts.put(stemmed+"|"+mods[2], sentPOCCounts);
			}
			Integer pocpocCounts = POCPOCCounts.get("</s>"+"|"+prevPOC);

			if(pocpocCounts == null) {
				pocpocCounts = 1;
			} else if (pocpocCounts != null) {
				pocpocCounts++;
			}

			POCPOCCounts.put("</s>"+"|"+prevPOC, pocpocCounts);

		}

		for(String poc: POCCounts.keySet()) {
			Integer denominator = POCCounts.get(poc);
			for(String key: wordPOCCounts.keySet()) {
				String splits[] = key.split("\\|");
				if(splits[1].equals(poc)) {
					Integer numerator = wordPOCCounts.get(key);
					Double value = numerator.doubleValue()/denominator.doubleValue();
					wordPOCProbs.put(key, value);					
				}
			}
		}

		for(String poc: POCCounts.keySet()) {
			Integer denominator = POCCounts.get(poc);
			for(String key: sentencePOCCounts.keySet()) {
				String splits[] = key.split("\\|");
				if(splits[1].equals(poc)) {
					Integer numerator = sentencePOCCounts.get(key);
					Double value = numerator.doubleValue()/denominator.doubleValue();
					sentencePOCProbs.put(key, value);					
				}
			}
		}

		// Populate POC probs & POC|POC probs
		Double sumPOCs = 0.0;
		for(String POC: POCCounts.keySet()) {
			sumPOCs += POCCounts.get(POC); 
		}

		for(String POC: POCCounts.keySet()) {
			POCProbs.put(POC, POCCounts.get(POC)/sumPOCs);			
		}

		for(String pocpoc: POCPOCCounts.keySet()) {
			String poc = pocpoc.split("\\|")[1];
			Double count = POCPOCCounts.get(pocpoc).doubleValue();
			Double totalCount = 0.0;
			if (POCCounts.get(poc) == null) {
				totalCount = 1.0;
			} else {
				totalCount = POCCounts.get(poc).doubleValue();
			}

			POCPOCProbs.put(pocpoc, count/totalCount);
		}

		for(String line : temp) {
			FileUtils.writeToFileAlreadyExisting(stemmedToCompleteURL, line);
		}
	}

}